package com.financing.platform.util;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.httpclient.params.DefaultHttpParams;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import org.aspectj.weaver.patterns.ParserException;

import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
import com.gargoylesoftware.htmlunit.Page;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.DomText;
import com.gargoylesoftware.htmlunit.html.HtmlAnchor;
import com.gargoylesoftware.htmlunit.html.HtmlDivision;
import com.gargoylesoftware.htmlunit.html.HtmlElement;
import com.gargoylesoftware.htmlunit.html.HtmlHeading1;
import com.gargoylesoftware.htmlunit.html.HtmlImage;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.html.HtmlSpan;
import com.gargoylesoftware.htmlunit.html.HtmlStrong;
import com.gargoylesoftware.htmlunit.html.HtmlTextInput;

/**
 * 文档处理类
 * 
 * @author huqiyun
 * 
 */
public class HtmlUnitFetchUtil {
	private static final Logger LOGGER = Logger
			.getLogger(HtmlUnitFetchUtil.class);
	private static WebClient client = null;
	private static SimpleDateFormat dateFormat1 = new SimpleDateFormat(
			"yyyy/MM/dd/");
	
	/**
	 * 获得Html 页面
	 * 
	 * @param url
	 *            链接
	 * @return
	 * @throws IOException
	 * @throws MalformedURLException
	 * @throws FailingHttpStatusCodeException
	 * @throws IOException
	 *             流异常
	 * @throws UnsupportedEncodingException
	 *             不支持的编码
	 * @throws ParserException
	 *             转换成XML结构异常
	 */
	public static HtmlElement getHtmlParset(String url) {
		try {
			if (client == null) {
				initClient();
			}else{
				client.closeAllWindows();
			}
			HtmlElement page = ((HtmlPage) client.getPage(url)).getDocumentElement();
			client.closeAllWindows();
			return page;
		} catch (Exception e) {
			LOGGER.error("get url error:" + url + " \t error message :"
					+ e.getMessage());
			return null;
		}
	}

	public static void initClient() {
		LOGGER.info("openWebClient");
		client = new WebClient(BrowserVersion.INTERNET_EXPLORER_8);
		// client = new
		// WebClient(BrowserVersion.CHROME,"183.218.63.168",8118);//代理配置
		client.getOptions().setThrowExceptionOnScriptError(false);// js错误是否直接抛出
		client.getOptions().setJavaScriptEnabled(false);// 是否加载js
		client.getOptions().setCssEnabled(false);// 是否加载css
		client.waitForBackgroundJavaScript(1000);
		client.setJavaScriptTimeout(1000);
		client.getOptions().setPrintContentOnFailingStatusCode(false);
		client.setAjaxController(new NicelyResynchronizingAjaxController());
		client.getOptions().setTimeout(35000); 
		DefaultHttpParams.getDefaultParams().setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true);
	}
	
	public static void destory() {
		if(client!=null){
			client.closeAllWindows();
			client=null;
		}
	}
	
	public static void setClientBrowserVersion(BrowserVersion version) {
		LOGGER.info("openWebClient");
		client = new WebClient(version);
		client.getOptions().setThrowExceptionOnScriptError(false);// js错误是否直接抛出
		client.getOptions().setJavaScriptEnabled(false);// 是否加载js
		client.getOptions().setCssEnabled(false);// 是否加载css
		client.waitForBackgroundJavaScript(1000);
		client.setJavaScriptTimeout(1000);
		client.getOptions().setPrintContentOnFailingStatusCode(false);
		client.setAjaxController(new NicelyResynchronizingAjaxController());
		client.getOptions().setTimeout(35000); 
		DefaultHttpParams.getDefaultParams().setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true);
	}

	public static void initClient(boolean openJs) {
		LOGGER.info("openWebClient");
		client = new WebClient(BrowserVersion.INTERNET_EXPLORER_8);
		// client = new
		// WebClient(BrowserVersion.CHROME,"183.218.63.168",8118);//代理配置
		client.getOptions().setThrowExceptionOnScriptError(false);// js错误是否直接抛出
		client.getOptions().setJavaScriptEnabled(openJs);// 是否加载js
		client.getOptions().setCssEnabled(false);// 是否加载css
		client.waitForBackgroundJavaScript(1000);
		client.setJavaScriptTimeout(1000);
		client.getOptions().setPrintContentOnFailingStatusCode(false);
		client.setAjaxController(new NicelyResynchronizingAjaxController());
		client.getOptions().setTimeout(35000);
		DefaultHttpParams.getDefaultParams().setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true);
	}

	/**
	 * 获得Html 页面
	 * 
	 * @param url
	 *            链接
	 * @return
	 * @throws IOException
	 * @throws MalformedURLException
	 * @throws FailingHttpStatusCodeException
	 * @throws IOException
	 *             流异常
	 * @throws UnsupportedEncodingException
	 *             不支持的编码
	 * @throws ParserException
	 *             转换成XML结构异常
	 */
	public static HtmlPage getHtml(String url) {
		try {
			if (client == null) {
				initClient();
			}else{
				client.closeAllWindows();
			}
			return (HtmlPage) client.getPage(url);
		} catch (Exception e) {
			if(e.getMessage().contains("403 Forbidden")){
				try {
					setClientBrowserVersion(BrowserVersion.INTERNET_EXPLORER_9);
					HtmlPage hp = (HtmlPage) client.getPage(url);
					initClient();
					client.closeAllWindows();
					return hp;
				} catch (Exception e1) {
					LOGGER.error("get url error:" + url + " \t error message :"
							+ e1.getMessage());
				} 
			}
			LOGGER.error("get url error:" + url + " \t error message :"
					+ e.getMessage());
			return null;
		}
	}

	/**
	 * 根据 标签名排除node
	 * 
	 * @param nodeList
	 * @param cssSelect
	 * @return
	 */
	@SuppressWarnings("unchecked")
	public static void debarNodeByCssSelect(HtmlElement element,
			String cssSelect) {
		List<HtmlElement> els = (List<HtmlElement>) element
				.getByXPath(getXpathByCssSelect(cssSelect));
		if (els != null && !els.isEmpty()) {
			for (int i = 0; i < els.size(); i++) {
				els.get(i).getParentNode().removeChild(els.get(i));
			}
		}
	}

	/**
	 * 去除文字链
	 * 
	 * @param nodeList
	 */
	@SuppressWarnings("unchecked")
	public static void debarLinkNode(HtmlElement element) {
		List<HtmlAnchor> els = (List<HtmlAnchor>) element.getByXPath(".//a");
		for (int i = 0; i < els.size(); i++) {
			els.get(i).removeAttribute("href");
			els.get(i).removeAttribute("target");
		}
	}

	/**
	 * 
	 * 
	 * @Title: download
	 * 
	 * @Description:下载图片 日期定义
	 * 
	 * @param @param location
	 * @param @param remota
	 * @param @return
	 * @param @throws IOException 设定文件
	 * 
	 * @return String 返回类型
	 * 
	 * @throws
	 */
	public static String download(String location, String remota)
			throws IOException {
		String endStr = remota.substring(remota.lastIndexOf("."));
		String dateString = dateFormat1.format(new Date());
		String fileName = dateString + System.currentTimeMillis() + endStr;
		File dirFile = new File(location + dateString);
		if (!dirFile.exists()) {
			dirFile.mkdirs();
		}
		String local = location + fileName;
		URL url = new URL(remota);
		HttpURLConnection conn = (HttpURLConnection) url.openConnection();
		BufferedInputStream bis = new BufferedInputStream(conn.getInputStream());
		BufferedOutputStream bos = new BufferedOutputStream(
				new FileOutputStream(new File(local)));
		byte[] bys = new byte[204800];
		int length = -1;
		while ((length = bis.read(bys, 0, bys.length)) != -1) {
			bos.write(bys, 0, length);
			bos.flush();
		}
		bos.flush();
		bos.close();
		bis.close();
		conn.disconnect();
		return fileName;
	}

	/**
	 * 
	 * 
	 * @Title: downloadByName
	 * 
	 * @Description: 下载文件
	 * 
	 * @param @param location
	 * @param @param remota
	 * @param @return
	 * @param @throws IOException 设定文件
	 * 
	 * @return String 返回类型
	 * 
	 * @throws
	 */
	public static String downloadByName(String location, String remota)
			throws IOException {
		location = location.replace("\\", "/");
		File dirFile = new File(
				location.substring(0, location.lastIndexOf("/")));
		if (!dirFile.exists()) {
			dirFile.mkdirs();
		}
		File file = new File(location);
		URL url = new URL(remota);
		HttpURLConnection conn = (HttpURLConnection) url.openConnection();
		IOUtils.copy(conn.getInputStream(), new FileOutputStream(file));
		return location;
	}


	/**
	 * 
	 * 
	 * @Title: downloadByName
	 * 
	 * @Description: 下载文件
	 * 
	 * @param @param location
	 * @param @param remota
	 * @param @return
	 * @param @throws IOException 设定文件
	 * 
	 * @return String 返回类型
	 * 
	 * @throws
	 */
	public static String downloadByWebClient(String location, String remota)
			throws IOException {
		location = location.replace("\\", "/");
		File dirFile = new File(
				location.substring(0, location.lastIndexOf("/")));
		if (!dirFile.exists()) {
			dirFile.mkdirs();
		}
		File file = new File(location);
		Page page = client.getPage(remota);
		IOUtils.copy(page.getWebResponse().getContentAsStream(),
				new FileOutputStream(file));
		return location;
	}
	/**
	 * 
	 * 
	 * @Title: downloadByName
	 * 
	 * @Description: 下载文件
	 * 
	 * @param @param location
	 * @param @param remota
	 * @param @return
	 * @param @throws IOException 设定文件
	 * 
	 * @return String 返回类型
	 * 
	 * @throws
	 */
	public static String downloadByWebClient(String location,HtmlElement element)
			throws IOException {
		location = location.replace("\\", "/");
		File dirFile = new File(
				location.substring(0, location.lastIndexOf("/")));
		if (!dirFile.exists()) {
			dirFile.mkdirs();
		}
		File file = new File(location);
		Page page = element.click();
		IOUtils.copy(page.getWebResponse().getContentAsStream(),
				new FileOutputStream(file));
		return page.getUrl().toString();
	}
	/**
	 * 
	 * 
	 * @Title: downloadByName
	 * 
	 * @Description: 下载文件
	 * 
	 * @param @param location
	 * @param @param remota
	 * @param @return
	 * @param @throws IOException 设定文件
	 * 
	 * @return String 返回类型
	 * 
	 * @throws
	 */
	public static String downloadByWebClient(String location,HtmlElement element,Set<String> old)
			throws IOException {
		location = location.replace("\\", "/");
		File dirFile = new File(
				location.substring(0, location.lastIndexOf("/")));
		if (!dirFile.exists()) {
			dirFile.mkdirs();
		}
		File file = new File(location);
		Page page = element.click();
		if(old.contains(page.getUrl().toString())){
			return "";
		}
		IOUtils.copy(page.getWebResponse().getContentAsStream(),
				new FileOutputStream(file));
		return page.getUrl().toString();
	}
	

	@SuppressWarnings("unchecked")
	public static List<HtmlElement> getElementByJs(HtmlElement element,
			String selStr) {
		List<HtmlElement> els = (List<HtmlElement>) element
				.getByXPath(getXpathByCssSelect(selStr));
		if (els == null || els.isEmpty()) {
			return null;
		} else {
			return els;
		}
	}

	/**
	 * 将Css选择器 转换成Xpath选择器
	 * 
	 * @param sel
	 * @return
	 */
	public static String getXpathByCssSelect(String sel) {
		if (sel.startsWith("/")) {
			return sel;
		}
		String[] arrs = sel.split(" ");
		StringBuffer xpath = new StringBuffer("./");
		for (String str : arrs) {
			str = str.replace("@", " ");
			if (str.startsWith(".")) {
				xpath.append("/*[@class='" + str.substring(1) + "']");
			} else if (str.startsWith("#")) {
				xpath.append("/*[@id='" + str.substring(1) + "']");
			} else if (str.startsWith("[")) {
				xpath.append(str);
			} else {
				xpath.append("/" + str);
			}
		}
		return xpath.toString();
	}

	public static void closeClient() {
		if (client != null) {
			LOGGER.info("closeWebClient");
			client.closeAllWindows();
			client = null;
		}
	}

	/**
	 * 
	 * 
	 * @Title: getText
	 * 
	 * @Description: 获取元素数据
	 * 
	 * @param @param licaishiinfo
	 * @param @param xpath
	 * @param @return 设定文件
	 * 
	 * @return String 返回类型
	 * 
	 * @throws
	 */
	public static String getText(HtmlElement licaishiinfo, String xpath) {
		Object o = licaishiinfo.getFirstByXPath(xpath);
		if (o == null) {
			return "";
		} else if (o instanceof HtmlStrong) {
			return ((HtmlStrong) o).getTextContent();
		} else if (o instanceof HtmlTextInput) {
			return ((HtmlTextInput) o).getTextContent();
		} else if (o instanceof HtmlSpan) {
			return ((HtmlSpan) o).getTextContent();
		} else if (o instanceof HtmlImage) {
			return ((HtmlImage) o).getSrcAttribute();
		} else if (o instanceof DomText) {
			return ((DomText) o).getTextContent();
		} else if (o instanceof HtmlDivision) {
			return ((HtmlDivision) o).getTextContent();
		} else if (o instanceof HtmlHeading1) {
			return ((HtmlHeading1) o).getTextContent();
		} else if (o instanceof HtmlAnchor) {
			return ((HtmlAnchor) o).getTextContent();
		}
		return (String) o;
	}
	public static String concatHost(String host, String url) {
		if (StringUtils.isBlank(host) || url.matches("(http|https).*?")) {
			return url;
		}
		host = host.toLowerCase();
		if (host.contains(".cn")) {
			host = host.substring(0, (host.indexOf(".cn") + 3));
		} else if (host.contains(".com")) {
			host = host.substring(0, (host.indexOf(".com") + 4));
		} else if (host.contains(".net")) {
			host = host.substring(0, (host.indexOf(".net") + 4));
		}
		url = (host + url).replaceAll("\\\\", "/");

		return url;
	}

	/**
	 * 
	 * 
	 * @Title: fecthAll
	 * 
	 * @Description:正则获取全部符合条件的数据 ;隔开
	 * 
	 * @param @param xml
	 * @param @param regex
	 * @param @param group
	 * @param @return 设定文件
	 * 
	 * @return String 返回类型
	 * 
	 * @throws
	 */
	public static String fecthAll(String xml, String regex, int group) {
		Pattern p = Pattern.compile(regex);
		String value = "";
		Matcher m = p.matcher(xml);
		while (m.find() && null != m.group(1)) {
			if (StringUtils.isBlank(value)) {
				value += m.group(group);
			} else {
				value += ";" + m.group(group);
			}
		}
		return value;
	}

	/**
	 * 
	 * 
	 * @Title: fecth
	 * 
	 * @Description:匹配首个符合正则的数据
	 * 
	 * @param @param xml
	 * @param @param regex
	 * @param @param group
	 * @param @return 设定文件
	 * 
	 * @return String 返回类型
	 * 
	 * @throws
	 */
	public static String fecth(String xml, String regex, int group) {
		Pattern p = Pattern.compile(regex);
		String value = "";
		Matcher m = p.matcher(xml);
		if (m.find() && null != m.group(1)) {
			value = m.group(group);
		}
		return value;
	}

}
